{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# 要点\n",
    "1. 组合使用 `evaluate.combine([\"metric1\", \"metric2\"])`\n",
    "2. 迭代计算 `add_batch` 和 `compute`"
   ],
   "id": "9fe7d6e4fa7f4f70"
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:21.778040Z",
     "start_time": "2025-05-20T12:36:17.584758Z"
    }
   },
   "source": "import evaluate",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\51165\\.conda\\envs\\e12\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:25.128567Z",
     "start_time": "2025-05-20T12:36:21.877012Z"
    }
   },
   "cell_type": "code",
   "source": [
    "aa = evaluate.list_evaluation_modules()\n",
    "print(aa)"
   ],
   "id": "10a22ffa3b27ea7f",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['lvwerra/test', 'angelina-wang/directional_bias_amplification', 'cpllab/syntaxgym', 'lvwerra/bary_score', 'hack/test_metric', 'yzha/ctc_eval', 'codeparrot/apps_metric', 'mfumanelli/geometric_mean', 'daiyizheng/valid', 'erntkn/dice_coefficient', 'mgfrantz/roc_auc_macro', 'Vlasta/pr_auc', 'gorkaartola/metric_for_tp_fp_samples', 'idsedykh/metric', 'idsedykh/codebleu2', 'idsedykh/codebleu', 'idsedykh/megaglue', 'Vertaix/vendiscore', 'GMFTBY/dailydialogevaluate', 'GMFTBY/dailydialog_evaluate', 'jzm-mailchimp/joshs_second_test_metric', 'ola13/precision_at_k', 'yulong-me/yl_metric', 'abidlabs/mean_iou', 'abidlabs/mean_iou2', 'KevinSpaghetti/accuracyk', 'NimaBoscarino/weat', 'ronaldahmed/nwentfaithfulness', 'Viona/infolm', 'kyokote/my_metric2', 'kashif/mape', 'Ochiroo/rouge_mn', 'leslyarun/fbeta_score', 'anz2/iliauniiccocrevaluation', 'zbeloki/m2', 'xu1998hz/sescore', 'dvitel/codebleu', 'NCSOFT/harim_plus', 'JP-SystemsX/nDCG', 'sportlosos/sescore', 'Drunper/metrica_tesi', 'jpxkqx/peak_signal_to_noise_ratio', 'jpxkqx/signal_to_reconstruction_error', 'hpi-dhc/FairEval', 'lvwerra/accuracy_score', 'ybelkada/cocoevaluate', 'harshhpareek/bertscore', 'posicube/mean_reciprocal_rank', 'bstrai/classification_report', 'omidf/squad_precision_recall', 'Josh98/nl2bash_m', 'BucketHeadP65/confusion_matrix', 'BucketHeadP65/roc_curve', 'yonting/average_precision_score', 'transZ/test_parascore', 'transZ/sbert_cosine', 'hynky/sklearn_proxy', 'xu1998hz/sescore_english_mt', 'xu1998hz/sescore_german_mt', 'xu1998hz/sescore_english_coco', 'xu1998hz/sescore_english_webnlg', 'unnati/kendall_tau_distance', 'Viona/fuzzy_reordering', 'Viona/kendall_tau', 'lhy/hamming_loss', 'lhy/ranking_loss', 'Muennighoff/code_eval_octopack', 'yuyijiong/quad_match_score', 'AlhitawiMohammed22/CER_Hu-Evaluation-Metrics', 'Yeshwant123/mcc', 'phonemetransformers/segmentation_scores', 'sma2023/wil', 'chanelcolgate/average_precision', 'ckb/unigram', 'Felipehonorato/eer', 'manueldeprada/beer', 'shunzh/apps_metric', 'He-Xingwei/sari_metric', 'langdonholmes/cohen_weighted_kappa', 'fschlatt/ner_eval', 'hyperml/balanced_accuracy', 'brian920128/doc_retrieve_metrics', 'guydav/restrictedpython_code_eval', 'k4black/codebleu', 'Natooz/ece', 'ingyu/klue_mrc', 'Vipitis/shadermatch', 'gabeorlanski/bc_eval', 'jjkim0807/code_eval', 'repllabs/mean_reciprocal_rank', 'repllabs/mean_average_precision', 'mtc/fragments', 'DarrenChensformer/eval_keyphrase', 'kdudzic/charmatch', 'Vallp/ter', 'DarrenChensformer/relation_extraction', 'Ikala-allen/relation_extraction', 'danieldux/hierarchical_softmax_loss', 'nlpln/tst', 'bdsaglam/jer', 'davebulaval/meaningbert', 'fnvls/bleu1234', 'fnvls/bleu_1234', 'nevikw39/specificity', 'yqsong/execution_accuracy', 'shalakasatheesh/squad_v2', 'arthurvqin/pr_auc', 'd-matrix/dmx_perplexity', 'akki2825/accents_unplugged_eval', 'juliakaczor/accents_unplugged_eval', 'Vickyage/accents_unplugged_eval', 'Qui-nn/accents_unplugged_eval', 'TelEl/accents_unplugged_eval', 'livvie/accents_unplugged_eval', 'DaliaCaRo/accents_unplugged_eval', 'alvinasvk/accents_unplugged_eval', 'LottieW/accents_unplugged_eval', 'sorgfresser/valid_efficiency_score', 'Fritz02/execution_accuracy', 'huanghuayu/multiclass_brier_score', 'jialinsong/apps_metric', 'DoctorSlimm/bangalore_score', 'agkphysics/ccc', 'DoctorSlimm/kaushiks_criteria', 'CZLC/rouge_raw', 'bascobasculino/mot-metrics', 'SEA-AI/mot-metrics', 'SEA-AI/det-metrics', 'saicharan2804/my_metric', 'red1bluelost/evaluate_genericify_cpp', 'maksymdolgikh/seqeval_with_fbeta', 'Bekhouche/NED', 'danieldux/isco_hierarchical_accuracy', 'ginic/phone_errors', 'berkatil/map', 'DarrenChensformer/action_generation', 'buelfhood/fbeta_score', 'danasone/ru_errant', 'helena-balabin/youden_index', 'SEA-AI/panoptic-quality', 'SEA-AI/box-metrics', 'MathewShen/bleu', 'berkatil/mrr', 'BridgeAI-Lab/SemF1', 'SEA-AI/horizon-metrics', 'maysonma/lingo_judge_metric', 'dannashao/span_metric', 'Aye10032/loss_metric', 'ag2435/my_metric', 'kilian-group/arxiv_score', 'bomjin/code_eval_octopack', 'svenwey/logmetric', 'bowdbeg/matching_series', 'BridgeAI-Lab/Sem-nCG', 'bowdbeg/patch_series', 'venkatasg/gleu', 'kbmlcoding/apps_metric', 'jijihuny/ecqa', 'prajwall/mse', 'd-matrix/dmxMetric', 'dotkaio/competition_math', 'bowdbeg/docred', 'Remeris/rouge_ru', 'jarod0411/aucpr', 'Ruchin/jaccard_similarity', 'phucdev/blanc_score', 'NathanMad/bertscore-with-torch_dtype', 'cointegrated/blaser_2_0_qe', 'ahnyeonchan/Alignment-and-Uniformity', 'Baleegh/Fluency_Score', 'mdocekal/multi_label_precision_recall_accuracy_fscore', 'phucdev/vihsd', 'argmaxinc/detailed-wer', 'SEA-AI/user-friendly-metrics', 'hage2000/code_eval_stdio', 'hage2000/my_metric', 'Natooz/levenshtein', 'Khaliq88/execution_accuracy', 'pico-lm/perplexity', 'mtzig/cross_entropy_loss', 'kiracurrie22/precision', 'openpecha/bleurt', 'SEA-AI/ref-metrics', 'Natooz/mse', 'buelfhood/fbeta_score_2', 'murinj/hter', 'pico-lm/blimp', 'nobody4/waf_metric', 'mdocekal/precision_recall_fscore_accuracy', 'Glazkov/mars', 'Aye10032/top5_error_rate', 'nhop/L3Score', 'maryxm/code_eval', 'Keanu256/meteor', 'sign/signwriting_similarity', 'maqiuping59/table_markdown', 'ncoop57/levenshtein_distance', 'kaleidophon/almost_stochastic_order', 'NeuraFusionAI/Arabic-Evaluation', 'lvwerra/element_count', 'prb977/cooccurrence_count', 'NimaBoscarino/pseudo_perplexity', 'ybelkada/toxicity', 'ronaldahmed/ccl_win', 'christopher/tokens_per_byte', 'lsy641/distinct', 'grepLeigh/perplexity', 'Charles95/element_count', 'Charles95/accuracy', 'Lucky28/honest']\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:27.647367Z",
     "start_time": "2025-05-20T12:36:25.130586Z"
    }
   },
   "cell_type": "code",
   "source": [
    "accuracy = evaluate.load(\"accuracy\")\n",
    "print(accuracy.__doc__)"
   ],
   "id": "34c499ecd94dd2b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with:\n",
      "Accuracy = (TP + TN) / (TP + TN + FP + FN)\n",
      " Where:\n",
      "TP: True positive\n",
      "TN: True negative\n",
      "FP: False positive\n",
      "FN: False negative\n",
      "\n",
      "Args:\n",
      "    predictions (`list` of `int`): Predicted labels.\n",
      "    references (`list` of `int`): Ground truth labels.\n",
      "    normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True.\n",
      "    sample_weight (`list` of `float`): Sample weights Defaults to None.\n",
      "\n",
      "Returns:\n",
      "    accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy.\n",
      "\n",
      "Examples:\n",
      "\n",
      "    Example 1-A simple example\n",
      "        >>> accuracy_metric = evaluate.load(\"accuracy\")\n",
      "        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])\n",
      "        >>> print(results)\n",
      "        {'accuracy': 0.5}\n",
      "\n",
      "    Example 2-The same as Example 1, except with `normalize` set to `False`.\n",
      "        >>> accuracy_metric = evaluate.load(\"accuracy\")\n",
      "        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False)\n",
      "        >>> print(results)\n",
      "        {'accuracy': 3.0}\n",
      "\n",
      "    Example 3-The same as Example 1, except with `sample_weight` set.\n",
      "        >>> accuracy_metric = evaluate.load(\"accuracy\")\n",
      "        >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])\n",
      "        >>> print(results)\n",
      "        {'accuracy': 0.8778625954198473}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "全局计算",
   "id": "4d142d56bf474e12"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:27.689201Z",
     "start_time": "2025-05-20T12:36:27.659199Z"
    }
   },
   "cell_type": "code",
   "source": [
    "results1 = accuracy.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])\n",
    "results2 = accuracy.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False)\n",
    "results22 = accuracy.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=True)\n",
    "results3 = accuracy.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4])\n",
    "results1, results2, results22, results3"
   ],
   "id": "6ee829775730fd19",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'accuracy': 0.5},\n",
       " {'accuracy': 3.0},\n",
       " {'accuracy': 0.5},\n",
       " {'accuracy': 0.8778625954198473})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "迭代计算",
   "id": "7043ff03b00a4c1e"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:27.710189Z",
     "start_time": "2025-05-20T12:36:27.701927Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for ref, pred in zip([0, 1, 2, 0, 1, 2], [0, 1, 1, 2, 1, 0]):\n",
    "    accuracy.add(references=ref, predictions=pred)\n",
    "accuracy.compute()"
   ],
   "id": "22361ff45de754cb",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.5}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:27.744582Z",
     "start_time": "2025-05-20T12:36:27.731129Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for ref, pred in zip([[0, 1], [2, 0], [1, 2]], [[0, 1], [1, 2], [1, 0]]):\n",
    "    accuracy.add_batch(references=ref, predictions=pred)\n",
    "accuracy.compute()"
   ],
   "id": "af8c9fe8d003b2e4",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.5}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "多指标",
   "id": "d8a27bc4ce90637"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:36.909860Z",
     "start_time": "2025-05-20T12:36:27.765839Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from evaluate import *\n",
    "metrics = evaluate.combine([\"accuracy\", \"f1\", \"recall\", \"precision\"])\n",
    "results1 = metrics.compute(references=[0, 1, 1, 0, 1, 0], predictions=[0, 1, 1, 1, 1, 0])\n",
    "results1\n"
   ],
   "id": "38eeef3f8a5c1ac1",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.8333333333333334,\n",
       " 'f1': 0.8571428571428571,\n",
       " 'recall': 1.0,\n",
       " 'precision': 0.75}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:36:38.297149Z",
     "start_time": "2025-05-20T12:36:36.924894Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification, DataCollatorWithPadding\n",
    "from MyHelper import *\n",
    "from datasets import Dataset\n",
    "from torch.utils.data import  DataLoader\n",
    "import torch\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(Config.hfl_rbt3, trust_remote_code=True)\n",
    "model = model.cuda()\n",
    "tokenizer = AutoTokenizer.from_pretrained(Config.hfl_rbt3)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=2e-5)"
   ],
   "id": "a81b30aa16f47555",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at hfl/rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T12:37:08.775541Z",
     "start_time": "2025-05-20T12:36:42.760587Z"
    }
   },
   "cell_type": "code",
   "source": [
    "ds = Dataset.from_csv(\"data/ChnSentiCorp_htl_all.csv\")\n",
    "ds = ds.filter(lambda x: x[\"review\"] is not None)\n",
    "ds = ds.train_test_split(test_size=0.1)\n",
    "\n",
    "def process_function(examples, tokenizer=tokenizer):\n",
    "    out = tokenizer(examples[\"review\"], truncation=True, max_length=128)\n",
    "    out[\"labels\"] = examples[\"label\"]\n",
    "    return out\n",
    "\n",
    "ds = ds.map(process_function, batched=True, remove_columns=[\"review\", \"label\"])\n",
    "\n",
    "trainset, validset = ds[\"train\"], ds[\"test\"]\n",
    "trainloader = DataLoader(trainset, batch_size=32, shuffle=True, collate_fn=DataCollatorWithPadding(tokenizer=tokenizer))\n",
    "validloader = DataLoader(validset, batch_size=64, shuffle=False, collate_fn=DataCollatorWithPadding(tokenizer=tokenizer))"
   ],
   "id": "ef4e4875f5008596",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map:   0%|          | 0/6988 [00:25<?, ? examples/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
      "\u001B[31mKeyboardInterrupt\u001B[39m                         Traceback (most recent call last)",
      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[9]\u001B[39m\u001B[32m, line 10\u001B[39m\n\u001B[32m      7\u001B[39m     out[\u001B[33m\"\u001B[39m\u001B[33mlabels\u001B[39m\u001B[33m\"\u001B[39m] = examples[\u001B[33m\"\u001B[39m\u001B[33mlabel\u001B[39m\u001B[33m\"\u001B[39m]\n\u001B[32m      8\u001B[39m     \u001B[38;5;28;01mreturn\u001B[39;00m out\n\u001B[32m---> \u001B[39m\u001B[32m10\u001B[39m ds = \u001B[43mds\u001B[49m\u001B[43m.\u001B[49m\u001B[43mmap\u001B[49m\u001B[43m(\u001B[49m\u001B[43mprocess_function\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbatched\u001B[49m\u001B[43m=\u001B[49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mremove_columns\u001B[49m\u001B[43m=\u001B[49m\u001B[43m[\u001B[49m\u001B[33;43m\"\u001B[39;49m\u001B[33;43mreview\u001B[39;49m\u001B[33;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[33;43m\"\u001B[39;49m\u001B[33;43mlabel\u001B[39;49m\u001B[33;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m     12\u001B[39m trainset, validset = ds[\u001B[33m\"\u001B[39m\u001B[33mtrain\u001B[39m\u001B[33m\"\u001B[39m], ds[\u001B[33m\"\u001B[39m\u001B[33mtest\u001B[39m\u001B[33m\"\u001B[39m]\n\u001B[32m     13\u001B[39m trainloader = DataLoader(trainset, batch_size=\u001B[32m32\u001B[39m, shuffle=\u001B[38;5;28;01mTrue\u001B[39;00m, collate_fn=DataCollatorWithPadding(tokenizer=tokenizer))\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\.conda\\envs\\e12\\Lib\\site-packages\\datasets\\dataset_dict.py:887\u001B[39m, in \u001B[36mDatasetDict.map\u001B[39m\u001B[34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_names, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, desc)\u001B[39m\n\u001B[32m    883\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m cache_file_names \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[32m    884\u001B[39m     cache_file_names = {k: \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;28;01mfor\u001B[39;00m k \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m}\n\u001B[32m    885\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m DatasetDict(\n\u001B[32m    886\u001B[39m     {\n\u001B[32m--> \u001B[39m\u001B[32m887\u001B[39m         k: \u001B[43mdataset\u001B[49m\u001B[43m.\u001B[49m\u001B[43mmap\u001B[49m\u001B[43m(\u001B[49m\n\u001B[32m    888\u001B[39m \u001B[43m            \u001B[49m\u001B[43mfunction\u001B[49m\u001B[43m=\u001B[49m\u001B[43mfunction\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    889\u001B[39m \u001B[43m            \u001B[49m\u001B[43mwith_indices\u001B[49m\u001B[43m=\u001B[49m\u001B[43mwith_indices\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    890\u001B[39m \u001B[43m            \u001B[49m\u001B[43mwith_rank\u001B[49m\u001B[43m=\u001B[49m\u001B[43mwith_rank\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    891\u001B[39m \u001B[43m            \u001B[49m\u001B[43minput_columns\u001B[49m\u001B[43m=\u001B[49m\u001B[43minput_columns\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    892\u001B[39m \u001B[43m            \u001B[49m\u001B[43mbatched\u001B[49m\u001B[43m=\u001B[49m\u001B[43mbatched\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    893\u001B[39m \u001B[43m            \u001B[49m\u001B[43mbatch_size\u001B[49m\u001B[43m=\u001B[49m\u001B[43mbatch_size\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    894\u001B[39m \u001B[43m            \u001B[49m\u001B[43mdrop_last_batch\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdrop_last_batch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    895\u001B[39m \u001B[43m            \u001B[49m\u001B[43mremove_columns\u001B[49m\u001B[43m=\u001B[49m\u001B[43mremove_columns\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    896\u001B[39m \u001B[43m            \u001B[49m\u001B[43mkeep_in_memory\u001B[49m\u001B[43m=\u001B[49m\u001B[43mkeep_in_memory\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    897\u001B[39m \u001B[43m            \u001B[49m\u001B[43mload_from_cache_file\u001B[49m\u001B[43m=\u001B[49m\u001B[43mload_from_cache_file\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    898\u001B[39m \u001B[43m            \u001B[49m\u001B[43mcache_file_name\u001B[49m\u001B[43m=\u001B[49m\u001B[43mcache_file_names\u001B[49m\u001B[43m[\u001B[49m\u001B[43mk\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    899\u001B[39m \u001B[43m            \u001B[49m\u001B[43mwriter_batch_size\u001B[49m\u001B[43m=\u001B[49m\u001B[43mwriter_batch_size\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    900\u001B[39m \u001B[43m            \u001B[49m\u001B[43mfeatures\u001B[49m\u001B[43m=\u001B[49m\u001B[43mfeatures\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    901\u001B[39m \u001B[43m            \u001B[49m\u001B[43mdisable_nullable\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdisable_nullable\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    902\u001B[39m \u001B[43m            \u001B[49m\u001B[43mfn_kwargs\u001B[49m\u001B[43m=\u001B[49m\u001B[43mfn_kwargs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    903\u001B[39m \u001B[43m            \u001B[49m\u001B[43mnum_proc\u001B[49m\u001B[43m=\u001B[49m\u001B[43mnum_proc\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    904\u001B[39m \u001B[43m            \u001B[49m\u001B[43mdesc\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdesc\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    905\u001B[39m \u001B[43m        \u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    906\u001B[39m         \u001B[38;5;28;01mfor\u001B[39;00m k, dataset \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m.items()\n\u001B[32m    907\u001B[39m     }\n\u001B[32m    908\u001B[39m )\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\.conda\\envs\\e12\\Lib\\site-packages\\datasets\\arrow_dataset.py:560\u001B[39m, in \u001B[36mtransmit_format.<locals>.wrapper\u001B[39m\u001B[34m(*args, **kwargs)\u001B[39m\n\u001B[32m    553\u001B[39m self_format = {\n\u001B[32m    554\u001B[39m     \u001B[33m\"\u001B[39m\u001B[33mtype\u001B[39m\u001B[33m\"\u001B[39m: \u001B[38;5;28mself\u001B[39m._format_type,\n\u001B[32m    555\u001B[39m     \u001B[33m\"\u001B[39m\u001B[33mformat_kwargs\u001B[39m\u001B[33m\"\u001B[39m: \u001B[38;5;28mself\u001B[39m._format_kwargs,\n\u001B[32m    556\u001B[39m     \u001B[33m\"\u001B[39m\u001B[33mcolumns\u001B[39m\u001B[33m\"\u001B[39m: \u001B[38;5;28mself\u001B[39m._format_columns,\n\u001B[32m    557\u001B[39m     \u001B[33m\"\u001B[39m\u001B[33moutput_all_columns\u001B[39m\u001B[33m\"\u001B[39m: \u001B[38;5;28mself\u001B[39m._output_all_columns,\n\u001B[32m    558\u001B[39m }\n\u001B[32m    559\u001B[39m \u001B[38;5;66;03m# apply actual function\u001B[39;00m\n\u001B[32m--> \u001B[39m\u001B[32m560\u001B[39m out: Union[\u001B[33m\"\u001B[39m\u001B[33mDataset\u001B[39m\u001B[33m\"\u001B[39m, \u001B[33m\"\u001B[39m\u001B[33mDatasetDict\u001B[39m\u001B[33m\"\u001B[39m] = \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    561\u001B[39m datasets: List[\u001B[33m\"\u001B[39m\u001B[33mDataset\u001B[39m\u001B[33m\"\u001B[39m] = \u001B[38;5;28mlist\u001B[39m(out.values()) \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(out, \u001B[38;5;28mdict\u001B[39m) \u001B[38;5;28;01melse\u001B[39;00m [out]\n\u001B[32m    562\u001B[39m \u001B[38;5;66;03m# re-apply format to the output\u001B[39;00m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\.conda\\envs\\e12\\Lib\\site-packages\\datasets\\arrow_dataset.py:3073\u001B[39m, in \u001B[36mDataset.map\u001B[39m\u001B[34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)\u001B[39m\n\u001B[32m   3067\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m transformed_dataset \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[32m   3068\u001B[39m     \u001B[38;5;28;01mwith\u001B[39;00m hf_tqdm(\n\u001B[32m   3069\u001B[39m         unit=\u001B[33m\"\u001B[39m\u001B[33m examples\u001B[39m\u001B[33m\"\u001B[39m,\n\u001B[32m   3070\u001B[39m         total=pbar_total,\n\u001B[32m   3071\u001B[39m         desc=desc \u001B[38;5;129;01mor\u001B[39;00m \u001B[33m\"\u001B[39m\u001B[33mMap\u001B[39m\u001B[33m\"\u001B[39m,\n\u001B[32m   3072\u001B[39m     ) \u001B[38;5;28;01mas\u001B[39;00m pbar:\n\u001B[32m-> \u001B[39m\u001B[32m3073\u001B[39m \u001B[43m        \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mrank\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdone\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcontent\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mDataset\u001B[49m\u001B[43m.\u001B[49m\u001B[43m_map_single\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mdataset_kwargs\u001B[49m\u001B[43m)\u001B[49m\u001B[43m:\u001B[49m\n\u001B[32m   3074\u001B[39m \u001B[43m            \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mdone\u001B[49m\u001B[43m:\u001B[49m\n\u001B[32m   3075\u001B[39m \u001B[43m                \u001B[49m\u001B[43mshards_done\u001B[49m\u001B[43m \u001B[49m\u001B[43m+\u001B[49m\u001B[43m=\u001B[49m\u001B[43m \u001B[49m\u001B[32;43m1\u001B[39;49m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\.conda\\envs\\e12\\Lib\\site-packages\\datasets\\arrow_dataset.py:3476\u001B[39m, in \u001B[36mDataset._map_single\u001B[39m\u001B[34m(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)\u001B[39m\n\u001B[32m   3472\u001B[39m indices = \u001B[38;5;28mlist\u001B[39m(\n\u001B[32m   3473\u001B[39m     \u001B[38;5;28mrange\u001B[39m(*(\u001B[38;5;28mslice\u001B[39m(i, i + batch_size).indices(shard.num_rows)))\n\u001B[32m   3474\u001B[39m )  \u001B[38;5;66;03m# Something simpler?\u001B[39;00m\n\u001B[32m   3475\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m-> \u001B[39m\u001B[32m3476\u001B[39m     batch = \u001B[43mapply_function_on_filtered_inputs\u001B[49m\u001B[43m(\u001B[49m\n\u001B[32m   3477\u001B[39m \u001B[43m        \u001B[49m\u001B[43mbatch\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   3478\u001B[39m \u001B[43m        \u001B[49m\u001B[43mindices\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   3479\u001B[39m \u001B[43m        \u001B[49m\u001B[43mcheck_same_num_examples\u001B[49m\u001B[43m=\u001B[49m\u001B[38;5;28;43mlen\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mshard\u001B[49m\u001B[43m.\u001B[49m\u001B[43mlist_indexes\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\u001B[43m \u001B[49m\u001B[43m>\u001B[49m\u001B[43m \u001B[49m\u001B[32;43m0\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[32m   3480\u001B[39m \u001B[43m        \u001B[49m\u001B[43moffset\u001B[49m\u001B[43m=\u001B[49m\u001B[43moffset\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   3481\u001B[39m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m   3482\u001B[39m \u001B[38;5;28;01mexcept\u001B[39;00m NumExamplesMismatchError:\n\u001B[32m   3483\u001B[39m     \u001B[38;5;28;01mraise\u001B[39;00m DatasetTransformationNotAllowedError(\n\u001B[32m   3484\u001B[39m         \u001B[33m\"\u001B[39m\u001B[33mUsing `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn\u001B[39m\u001B[33m'\u001B[39m\u001B[33mt create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it.\u001B[39m\u001B[33m\"\u001B[39m\n\u001B[32m   3485\u001B[39m     ) \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\.conda\\envs\\e12\\Lib\\site-packages\\datasets\\arrow_dataset.py:3338\u001B[39m, in \u001B[36mDataset._map_single.<locals>.apply_function_on_filtered_inputs\u001B[39m\u001B[34m(pa_inputs, indices, check_same_num_examples, offset)\u001B[39m\n\u001B[32m   3336\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m with_rank:\n\u001B[32m   3337\u001B[39m     additional_args += (rank,)\n\u001B[32m-> \u001B[39m\u001B[32m3338\u001B[39m processed_inputs = \u001B[43mfunction\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43mfn_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43madditional_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mfn_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m   3339\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(processed_inputs, LazyDict):\n\u001B[32m   3340\u001B[39m     processed_inputs = {\n\u001B[32m   3341\u001B[39m         k: v \u001B[38;5;28;01mfor\u001B[39;00m k, v \u001B[38;5;129;01min\u001B[39;00m processed_inputs.data.items() \u001B[38;5;28;01mif\u001B[39;00m k \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m processed_inputs.keys_to_format\n\u001B[32m   3342\u001B[39m     }\n",
      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[9]\u001B[39m\u001B[32m, line 8\u001B[39m, in \u001B[36mprocess_function\u001B[39m\u001B[34m(examples, tokenizer)\u001B[39m\n\u001B[32m      6\u001B[39m out = tokenizer(examples[\u001B[33m\"\u001B[39m\u001B[33mreview\u001B[39m\u001B[33m\"\u001B[39m], truncation=\u001B[38;5;28;01mTrue\u001B[39;00m, max_length=\u001B[32m128\u001B[39m)\n\u001B[32m      7\u001B[39m out[\u001B[33m\"\u001B[39m\u001B[33mlabels\u001B[39m\u001B[33m\"\u001B[39m] = examples[\u001B[33m\"\u001B[39m\u001B[33mlabel\u001B[39m\u001B[33m\"\u001B[39m]\n\u001B[32m----> \u001B[39m\u001B[32m8\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mout\u001B[49m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m_pydevd_bundle\\\\pydevd_cython_win32_312_64.pyx:1187\u001B[39m, in \u001B[36m_pydevd_bundle.pydevd_cython_win32_312_64.SafeCallWrapper.__call__\u001B[39m\u001B[34m()\u001B[39m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m_pydevd_bundle\\\\pydevd_cython_win32_312_64.pyx:627\u001B[39m, in \u001B[36m_pydevd_bundle.pydevd_cython_win32_312_64.PyDBFrame.trace_dispatch\u001B[39m\u001B[34m()\u001B[39m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m_pydevd_bundle\\\\pydevd_cython_win32_312_64.pyx:937\u001B[39m, in \u001B[36m_pydevd_bundle.pydevd_cython_win32_312_64.PyDBFrame.trace_dispatch\u001B[39m\u001B[34m()\u001B[39m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m_pydevd_bundle\\\\pydevd_cython_win32_312_64.pyx:928\u001B[39m, in \u001B[36m_pydevd_bundle.pydevd_cython_win32_312_64.PyDBFrame.trace_dispatch\u001B[39m\u001B[34m()\u001B[39m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m_pydevd_bundle\\\\pydevd_cython_win32_312_64.pyx:585\u001B[39m, in \u001B[36m_pydevd_bundle.pydevd_cython_win32_312_64.PyDBFrame.do_wait_suspend\u001B[39m\u001B[34m()\u001B[39m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\AppData\\Local\\Programs\\PyCharm\\plugins\\python-ce\\helpers\\pydev\\pydevd.py:1220\u001B[39m, in \u001B[36mPyDB.do_wait_suspend\u001B[39m\u001B[34m(self, thread, frame, event, arg, send_suspend_message, is_unhandled_exception)\u001B[39m\n\u001B[32m   1217\u001B[39m         from_this_thread.append(frame_id)\n\u001B[32m   1219\u001B[39m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m._threads_suspended_single_notification.notify_thread_suspended(thread_id, stop_reason):\n\u001B[32m-> \u001B[39m\u001B[32m1220\u001B[39m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43m_do_wait_suspend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mthread\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mframe\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mevent\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43marg\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msuspend_type\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfrom_this_thread\u001B[49m\u001B[43m)\u001B[49m\n",
      "\u001B[36mFile \u001B[39m\u001B[32m~\\AppData\\Local\\Programs\\PyCharm\\plugins\\python-ce\\helpers\\pydev\\pydevd.py:1235\u001B[39m, in \u001B[36mPyDB._do_wait_suspend\u001B[39m\u001B[34m(self, thread, frame, event, arg, suspend_type, from_this_thread)\u001B[39m\n\u001B[32m   1232\u001B[39m             \u001B[38;5;28mself\u001B[39m._call_mpl_hook()\n\u001B[32m   1234\u001B[39m         \u001B[38;5;28mself\u001B[39m.process_internal_commands()\n\u001B[32m-> \u001B[39m\u001B[32m1235\u001B[39m         \u001B[43mtime\u001B[49m\u001B[43m.\u001B[49m\u001B[43msleep\u001B[49m\u001B[43m(\u001B[49m\u001B[32;43m0.01\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[32m   1237\u001B[39m \u001B[38;5;28mself\u001B[39m.cancel_async_evaluation(get_current_thread_id(thread), \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28mid\u001B[39m(frame)))\n\u001B[32m   1239\u001B[39m \u001B[38;5;66;03m# process any stepping instructions\u001B[39;00m\n",
      "\u001B[31mKeyboardInterrupt\u001B[39m: "
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "metrics = evaluate.combine([\"accuracy\", \"f1\", \"recall\", \"precision\"])\n",
    "\n",
    "def eval():\n",
    "    model.eval()\n",
    "    with torch.inference_mode():\n",
    "        for batch in validloader:\n",
    "            batch = {k: v.cuda() for k, v in batch.items()}\n",
    "            outputs = model(**batch)\n",
    "            pred = outputs.logits.argmax(dim=-1)\n",
    "            metrics.add_batch(batch[\"labels\"].long(), pred.long())\n",
    "    return metrics.compute()\n",
    "\n",
    "def train(epochs=3, log_step=30):\n",
    "    global_step = 0\n",
    "    for ep in range(epochs):\n",
    "        model.train()\n",
    "        for batch in trainloader:\n",
    "            batch = {k: v.cuda() for k, v in batch.items()}\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(**batch)\n",
    "            outputs[\"loss\"].backward()\n",
    "            optimizer.step()\n",
    "            if global_step % log_step == 0:\n",
    "                print(f\"epoch: {ep}, global_step: {global_step}, loss: {outputs['loss'].item()}\")\n",
    "            global_step += 1\n",
    "\n",
    "        mt = eval()\n",
    "        print(f\"epoch: {ep}, acc: {mt['accuracy']}, f1: {mt['f1']}, recall: {mt['recall']}, precision: {mt['precision']}\")\n",
    "\n",
    "train()"
   ],
   "id": "ca4318cc35addd93",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
